from bigdl.llm import optimize_model
from transformers import AutoTokenizer,AutoModel
model_path = r"D:\Code\GitCode\chatglm\chatglm3-6b"
model = AutoModel.from_pretrained(model_path,trust_remote_code=True)
model=optimize_model(model,low_bit="sym_int4")
model.save_low_bit(r"D:\Code\GitCode\chatglm\chatglm3-6b_lowbit")
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
tokenizer.save_pretrained(r"D:\Code\GitCode\chatglm\chatglm3-6b_lowbit_tokenizer")